#ifndef TOKENIZER_HPP
#define TOKENIZER_HPP

#include <string>
#include <vector>
#include <memory>

class Tokenizer {
public:
    virtual std::vector<int> encode(const std::string& text) = 0;
    virtual std::string decode(const std::vector<int>& tokens) = 0;
    virtual ~Tokenizer() = default;
};

std::unique_ptr<Tokenizer> create_tokenizer(const std::string &tok_path);

#endif // TOKENIZER_HPP
